package chapter06

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, SparkSession}

/**
 * 自定义udf函数
 */
object UDFFunctionApp {
  def main(args: Array[String]): Unit = {
      val spark: SparkSession = SparkSession.builder()
        .appName("UDFFunctionApp")
        .master("local")
        .getOrCreate()

    import spark.implicits._

    val InfoRDD: RDD[String] = spark.sparkContext
      .textFile("D:\\JAVApros\\spark_pk\\data\\hobbies.txt")

    val infoDF: DataFrame = InfoRDD.map(x => {
      val splits: Array[String] = x.split("###")
      Hobbies(splits(0), splits(1))
    }).toDF()

    //infoDF.show()
    //todo 自定义函数分为三步
    //todo 定义函数 注册函数
    spark.udf.register("hobby_num",(s:String)=>{
      s.split(",").size
    })

    //todo 函数的使用
    infoDF.createOrReplaceTempView("hobbies")

    spark.sql(
      """
        |SELECT
        |name,
        |hobbies,
        |hobby_num(hobbies) as hobby_num
        |FROM hobbies
        |""".stripMargin).show()



    spark.stop()

  }
  case class Hobbies(name:String,hobbies:String)

}
